{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型加载与保存"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:10:22.435062Z",
     "start_time": "2025-09-05T14:10:22.419030Z"
    }
   },
   "source": "from transformers import AutoConfig, AutoModel, AutoTokenizer, BertConfig",
   "outputs": [],
   "execution_count": 50
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在线加载"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:10:49.882987Z",
     "start_time": "2025-09-05T14:10:25.314832Z"
    }
   },
   "source": "model = AutoModel.from_pretrained(\"hfl/rbt3\", force_download = True)",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "config.json:   0%|          | 0.00/404 [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "9c950ca75cae45e3a43c9389765a2c56"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86134\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\huggingface_hub\\file_download.py:159: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\86134\\.cache\\huggingface\\hub\\models--hfl--rbt3. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "config.json:   0%|          | 0.00/404 [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "5063f466f37b4b11984d54b67950906e"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "data": {
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/156M [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d8cceecca2494c099656c9fd7edc3215"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 51
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型下载（需要科学）"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:10:58.325401Z",
     "start_time": "2025-09-05T14:10:58.217138Z"
    }
   },
   "source": [
    "!git clone \"https://huggingface.co/hfl/rbt3\""
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'rbt3' already exists and is not an empty directory.\n"
     ]
    }
   ],
   "execution_count": 52
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:00.885931Z",
     "start_time": "2025-09-05T14:11:00.455804Z"
    }
   },
   "source": [
    "!git lfs clone \"https://huggingface.co/hfl/rbt3\" --include=\"*.bin\""
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: `git lfs clone` is deprecated and will not be updated\n",
      "          with new flags from `git clone`\n",
      "\n",
      "`git clone` has been updated in upstream Git to have comparable\n",
      "speeds to `git lfs clone`.\n",
      "fatal: destination path 'rbt3' already exists and is not an empty directory.\n",
      "Error(s) during clone:\n",
      "`git clone` failed: exit status 128\n"
     ]
    }
   ],
   "execution_count": 53
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 离线加载"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:06.154698Z",
     "start_time": "2025-09-05T14:11:06.081695Z"
    }
   },
   "source": "model = AutoModel.from_pretrained(\"rbt3\")",
   "outputs": [],
   "execution_count": 54
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型加载参数"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:11.107928Z",
     "start_time": "2025-09-05T14:11:11.052587Z"
    }
   },
   "source": [
    "model = AutoModel.from_pretrained(\"rbt3\")"
   ],
   "outputs": [],
   "execution_count": 55
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:12.820399Z",
     "start_time": "2025-09-05T14:11:12.815397Z"
    }
   },
   "source": [
    "# 查看参数\n",
    "model.config"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertConfig {\n",
       "  \"_name_or_path\": \"rbt3\",\n",
       "  \"architectures\": [\n",
       "    \"BertForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"classifier_dropout\": null,\n",
       "  \"directionality\": \"bidi\",\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"bert\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 3,\n",
       "  \"output_past\": true,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"pooler_fc_size\": 768,\n",
       "  \"pooler_num_attention_heads\": 12,\n",
       "  \"pooler_num_fc_layers\": 3,\n",
       "  \"pooler_size_per_head\": 128,\n",
       "  \"pooler_type\": \"first_token_transform\",\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.43.4\",\n",
       "  \"type_vocab_size\": 2,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 21128\n",
       "}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 56
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:15.019058Z",
     "start_time": "2025-09-05T14:11:15.013037Z"
    }
   },
   "source": [
    "config = AutoConfig.from_pretrained(\"./rbt3/\")\n",
    "config"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertConfig {\n",
       "  \"_name_or_path\": \"./rbt3/\",\n",
       "  \"architectures\": [\n",
       "    \"BertForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"classifier_dropout\": null,\n",
       "  \"directionality\": \"bidi\",\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"bert\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 3,\n",
       "  \"output_past\": true,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"pooler_fc_size\": 768,\n",
       "  \"pooler_num_attention_heads\": 12,\n",
       "  \"pooler_num_fc_layers\": 3,\n",
       "  \"pooler_size_per_head\": 128,\n",
       "  \"pooler_type\": \"first_token_transform\",\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.43.4\",\n",
       "  \"type_vocab_size\": 2,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 21128\n",
       "}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 57
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:16.613323Z",
     "start_time": "2025-09-05T14:11:16.609046Z"
    }
   },
   "source": [
    "config.output_attentions"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 58
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:18.375552Z",
     "start_time": "2025-09-05T14:11:18.372552Z"
    }
   },
   "source": [
    "# ctrl+点击进入查看更多参数\n",
    "from transformers import  BertConfig"
   ],
   "outputs": [],
   "execution_count": 59
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型调用"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:19.755096Z",
     "start_time": "2025-09-05T14:11:19.725355Z"
    }
   },
   "source": [
    "sen = \"弱小的我也有大梦想！\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"rbt3\")\n",
    "inputs = tokenizer(sen, return_tensors = \"pt\")\n",
    "inputs"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[ 101, 2483, 2207, 4638, 2769,  738, 3300, 1920, 3457, 2682, 8013,  102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 60
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 不带Model Head的模型调用"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:24.246316Z",
     "start_time": "2025-09-05T14:11:24.188421Z"
    }
   },
   "source": "model = AutoModel.from_pretrained(\"rbt3\", output_attentions = True)",
   "outputs": [],
   "execution_count": 61
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:27.323423Z",
     "start_time": "2025-09-05T14:11:27.267714Z"
    }
   },
   "source": [
    "output = model(**inputs)\n",
    "output"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "BertSdpaSelfAttention is used but `torch.nn.functional.scaled_dot_product_attention` does not support non-absolute `position_embedding_type` or `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation=\"eager\"` when loading the model.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BaseModelOutputWithPoolingAndCrossAttentions(last_hidden_state=tensor([[[ 0.6804,  0.6664,  0.7170,  ..., -0.4102,  0.7839, -0.0262],\n",
       "         [-0.7378, -0.2748,  0.5034,  ..., -0.1359, -0.4331, -0.5874],\n",
       "         [-0.0212,  0.5642,  0.1032,  ..., -0.3617,  0.4646, -0.4747],\n",
       "         ...,\n",
       "         [ 0.0853,  0.6679, -0.1757,  ..., -0.0942,  0.4664,  0.2925],\n",
       "         [ 0.3336,  0.3224, -0.3355,  ..., -0.3262,  0.2532, -0.2507],\n",
       "         [ 0.6761,  0.6688,  0.7154,  ..., -0.4083,  0.7824, -0.0224]]],\n",
       "       grad_fn=<NativeLayerNormBackward0>), pooler_output=tensor([[-1.2646e-01, -9.8619e-01, -1.0000e+00, -9.8325e-01,  8.0238e-01,\n",
       "         -6.6268e-02,  6.6919e-02,  1.4784e-01,  9.9451e-01,  9.9995e-01,\n",
       "         -8.3051e-02, -1.0000e+00, -9.8865e-02,  9.9980e-01, -1.0000e+00,\n",
       "          9.9993e-01,  9.8291e-01,  9.5363e-01, -9.9948e-01, -1.3219e-01,\n",
       "         -9.9733e-01, -7.7934e-01,  1.0720e-01,  9.8040e-01,  9.9953e-01,\n",
       "         -9.9939e-01, -9.9997e-01,  1.4967e-01, -8.7627e-01, -9.9996e-01,\n",
       "         -9.9821e-01, -9.9999e-01,  1.9396e-01, -1.1277e-01,  9.9359e-01,\n",
       "         -9.9153e-01,  4.4752e-02, -9.8731e-01, -9.9942e-01, -9.9982e-01,\n",
       "          2.9360e-02,  9.9847e-01, -9.2012e-03,  9.9999e-01,  1.7111e-01,\n",
       "          4.5070e-03,  9.9998e-01,  9.9467e-01,  4.9727e-03, -9.0707e-01,\n",
       "          6.9056e-02, -1.8141e-01, -9.8831e-01,  9.9668e-01,  4.9800e-01,\n",
       "          1.2997e-01,  9.9895e-01, -1.0000e+00, -9.9990e-01,  9.9478e-01,\n",
       "         -9.9989e-01,  9.9906e-01,  9.9820e-01,  9.9990e-01, -6.8953e-01,\n",
       "          9.9990e-01,  9.9987e-01,  9.4563e-01, -3.7660e-01, -1.0000e+00,\n",
       "          1.3151e-01, -9.7371e-01, -9.9997e-01, -1.3228e-02, -2.9800e-01,\n",
       "         -9.9985e-01,  9.9662e-01, -2.0004e-01,  9.9997e-01,  3.6876e-01,\n",
       "         -9.9997e-01,  1.5462e-01,  1.9265e-01,  8.9872e-02,  9.9996e-01,\n",
       "          9.9998e-01,  1.5183e-01, -8.9714e-01, -2.1646e-01, -9.9922e-01,\n",
       "         -4.9491e-01,  9.9957e-01,  9.9998e-01, -9.9998e-01,  9.9995e-01,\n",
       "         -5.1678e-01,  5.2056e-02,  5.4613e-02, -9.9816e-01,  9.9328e-01,\n",
       "          1.2779e-04, -1.3744e-01,  1.0000e+00,  9.9984e-01, -3.4417e-01,\n",
       "         -9.9995e-01, -9.9573e-01,  9.9988e-01, -9.9981e-01,  6.3345e-02,\n",
       "          1.0000e+00,  9.4779e-01,  1.0000e+00,  9.9946e-01,  9.9999e-01,\n",
       "         -9.9999e-01, -4.3540e-01,  2.3526e-01, -9.9997e-01,  9.9905e-01,\n",
       "         -9.9272e-01,  1.4150e-01, -9.3078e-01, -8.8246e-02, -1.2646e-02,\n",
       "         -9.9999e-01,  1.8301e-02,  3.9717e-02, -9.8869e-01, -9.9944e-01,\n",
       "         -9.9975e-01, -9.9994e-01,  9.9785e-01,  7.9386e-01,  2.7185e-01,\n",
       "         -1.5316e-01,  9.0940e-02, -9.5427e-02, -1.0000e+00, -9.9974e-01,\n",
       "         -9.9999e-01,  9.5742e-01, -3.5169e-01,  9.9779e-01, -9.9894e-01,\n",
       "          9.9997e-01, -9.9997e-01,  9.9997e-01,  9.9414e-01, -2.7013e-01,\n",
       "         -9.7769e-01, -1.1832e-01, -9.9976e-01, -4.3268e-02,  2.7016e-02,\n",
       "          9.9011e-01,  9.9801e-01,  7.6135e-01, -9.8868e-01,  1.0000e+00,\n",
       "         -9.9946e-01,  9.7542e-01,  1.4210e-01,  9.9955e-01,  1.0000e+00,\n",
       "         -1.0000e+00,  2.5602e-01, -1.0000e+00,  6.9886e-01,  1.1957e-01,\n",
       "          9.9996e-01,  9.9962e-01,  9.7632e-01,  9.9998e-01, -8.6662e-01,\n",
       "         -9.9994e-01,  9.5777e-01, -1.0000e+00,  9.8048e-01,  1.0000e+00,\n",
       "          9.6255e-02,  5.4609e-01,  9.9999e-01, -6.1723e-01,  9.9141e-01,\n",
       "         -1.0398e-01, -1.9344e-01, -9.9981e-01,  2.0875e-01,  9.4846e-01,\n",
       "          9.9600e-01, -9.9833e-01, -3.6391e-02,  9.8665e-01, -3.1239e-02,\n",
       "          6.7723e-02, -9.9968e-01, -9.9970e-01,  9.9994e-01,  9.9983e-01,\n",
       "          6.2746e-01, -2.7500e-01,  1.0000e+00, -1.1557e-01,  9.9997e-01,\n",
       "         -7.4188e-02,  8.3064e-01, -8.6325e-02,  9.9989e-01,  1.6120e-01,\n",
       "          8.7417e-01,  4.2873e-03,  9.9993e-01, -8.4737e-01, -9.9999e-01,\n",
       "          8.9603e-02,  8.9435e-01,  1.0934e-01, -9.9877e-01,  2.1512e-01,\n",
       "         -4.4630e-01,  9.9997e-01,  1.9113e-01, -9.8081e-01,  9.9929e-01,\n",
       "         -9.9977e-01,  6.1149e-01, -1.0000e+00, -9.9892e-01,  9.9998e-01,\n",
       "         -2.9081e-01, -1.0000e+00,  8.6111e-01,  1.0000e+00, -8.8875e-01,\n",
       "          9.9958e-01, -2.4632e-01, -9.9994e-01, -1.4219e-02,  3.7028e-02,\n",
       "         -1.0000e+00, -9.9450e-01, -1.0000e+00, -8.2727e-01, -1.4345e-01,\n",
       "          9.9392e-01, -1.0000e+00,  1.1743e-01, -9.9999e-01,  9.9873e-01,\n",
       "          9.9997e-01, -1.5349e-01,  1.7382e-01,  1.0000e+00, -3.5095e-01,\n",
       "          1.3408e-01, -8.4305e-01,  3.7473e-01,  2.2783e-02,  9.9625e-01,\n",
       "          3.2440e-01,  9.9899e-01, -9.9979e-01,  2.4282e-01,  8.5080e-01,\n",
       "         -1.0000e+00, -1.0721e-01,  9.9331e-01,  2.8107e-02,  1.0824e-01,\n",
       "         -1.8632e-01,  1.7009e-01,  9.5663e-01,  9.9947e-01,  1.0000e+00,\n",
       "          9.9177e-01,  9.9999e-01,  9.9999e-01, -3.1200e-01, -9.9837e-01,\n",
       "         -5.6503e-01,  2.3465e-01, -1.0000e+00, -9.8613e-01, -9.9979e-01,\n",
       "          9.9075e-01,  1.1560e-01,  1.0000e+00, -1.0000e+00,  1.0000e+00,\n",
       "         -9.6587e-01,  8.5970e-02, -5.3796e-02,  1.2931e-01, -5.4356e-01,\n",
       "         -1.2560e-01,  9.9880e-01, -7.6849e-02,  9.9302e-01,  9.9631e-01,\n",
       "         -4.9745e-03, -2.4950e-01,  2.0312e-01, -2.2919e-01,  9.9857e-01,\n",
       "         -9.9750e-01,  9.9836e-01,  1.0468e-01,  9.9982e-01, -4.5313e-01,\n",
       "         -1.0000e+00,  9.9977e-01, -9.9988e-01, -5.4165e-01, -9.9991e-01,\n",
       "         -9.8466e-01,  9.0575e-02, -9.8760e-01,  7.2146e-01,  9.9684e-01,\n",
       "          2.2268e-01,  1.4701e-01, -9.9999e-01, -9.6879e-01,  9.9483e-01,\n",
       "          9.9992e-01, -9.9977e-01,  9.9892e-01,  9.9656e-01, -9.3349e-01,\n",
       "          2.5862e-01,  9.7359e-01, -9.9937e-01,  9.8777e-01, -9.9999e-01,\n",
       "          1.1818e-01,  9.9960e-01, -1.7951e-01, -9.9984e-01, -9.2495e-01,\n",
       "         -2.2660e-02,  7.8255e-01, -2.6023e-02,  9.9999e-01, -1.2445e-02,\n",
       "          1.5701e-01, -9.9998e-01, -9.9624e-01, -8.6672e-01,  3.4873e-01,\n",
       "          9.9931e-01, -9.9999e-01, -6.6310e-02,  9.9949e-01, -9.9926e-01,\n",
       "         -4.1633e-01,  4.3388e-02,  8.4618e-02, -8.7278e-02, -9.9765e-01,\n",
       "         -9.9999e-01, -9.9998e-01,  9.9993e-01,  1.0225e-01, -5.4263e-04,\n",
       "          9.9924e-01,  9.9998e-01,  9.9997e-01, -9.8936e-01,  9.3540e-01,\n",
       "          9.9986e-01, -3.1887e-01,  1.1548e-01, -9.8294e-01,  1.4084e-01,\n",
       "         -8.1032e-01, -9.9606e-01,  1.2704e-01,  2.7952e-01, -6.5889e-01,\n",
       "         -9.9392e-01,  9.9999e-01,  9.9994e-01,  1.0000e+00, -1.0210e-01,\n",
       "         -9.4733e-01,  8.3178e-01, -9.4359e-01, -9.9962e-01, -4.4847e-02,\n",
       "          9.9938e-01, -9.9812e-01,  1.7198e-01,  7.5851e-02, -9.4664e-01,\n",
       "          9.9917e-01, -9.9949e-01,  1.5547e-01, -1.0000e+00, -9.9998e-01,\n",
       "         -9.9998e-01,  1.0000e+00,  9.2369e-02, -1.2598e-01, -9.9929e-01,\n",
       "          1.0000e+00,  9.8569e-01, -9.6164e-01, -2.5984e-01,  9.9998e-01,\n",
       "         -4.7267e-01, -8.6810e-01, -1.0000e+00, -9.9985e-01,  9.9819e-01,\n",
       "          1.2791e-01,  9.9999e-01,  8.4013e-01, -9.9762e-01,  9.8651e-01,\n",
       "          9.7417e-01,  3.1610e-01, -9.9945e-01, -9.9936e-01, -3.3196e-03,\n",
       "          7.0084e-02,  1.5903e-01,  9.8475e-03, -5.9952e-02,  9.9992e-01,\n",
       "         -3.2020e-02, -9.5302e-02, -3.2294e-01,  1.0000e+00,  8.7427e-01,\n",
       "         -9.9866e-01, -6.7442e-01, -8.8977e-02, -9.9465e-01, -9.9605e-01,\n",
       "         -9.9996e-01, -2.6155e-01,  1.4165e-01,  4.0373e-02, -9.9220e-01,\n",
       "         -9.9825e-01, -9.9979e-01,  1.5166e-02, -9.9095e-01, -9.9897e-01,\n",
       "         -1.5214e-01, -9.9863e-01, -9.9987e-01,  9.9998e-01, -9.9994e-01,\n",
       "         -6.0612e-03,  9.7556e-01,  9.9935e-01, -9.9990e-01,  2.1632e-01,\n",
       "          9.9836e-01, -9.9856e-01,  2.3167e-01, -9.8543e-01, -2.4208e-03,\n",
       "         -8.9389e-01, -1.0000e+00,  3.3006e-01,  9.9995e-01,  9.9989e-01,\n",
       "          9.9846e-01,  9.1860e-01, -7.6863e-01,  9.6949e-01,  9.9988e-01,\n",
       "          1.0000e+00, -8.6599e-02, -1.3680e-01, -9.9999e-01, -6.0014e-01,\n",
       "          8.4746e-01,  1.0654e-01,  8.5102e-01, -9.9990e-01,  3.2253e-01,\n",
       "         -9.2435e-01,  2.9811e-01,  1.0000e+00,  9.9763e-01,  1.9317e-01,\n",
       "         -1.0000e+00,  2.9202e-01, -5.9693e-01,  2.1347e-01, -9.9327e-01,\n",
       "         -1.2919e-02,  1.0000e+00, -9.6718e-01,  2.9585e-01, -9.9959e-01,\n",
       "         -9.9972e-01,  9.9999e-01, -9.9997e-01,  9.9999e-01,  9.3987e-01,\n",
       "         -9.9433e-01, -2.2040e-01, -9.8471e-01, -5.8420e-02, -1.5101e-02,\n",
       "         -1.0072e-01, -3.6830e-01, -8.9496e-02, -1.0000e+00,  1.3947e-01,\n",
       "          9.8818e-01,  3.1653e-02, -1.7422e-01, -9.9989e-01,  2.3496e-01,\n",
       "          9.7184e-01, -9.9993e-01, -9.9996e-01, -1.7622e-01, -4.5059e-01,\n",
       "          1.3080e-01, -1.9689e-02, -4.3433e-02, -1.0663e-02, -9.9764e-01,\n",
       "          5.8259e-02,  9.8391e-01, -9.9202e-01,  8.4339e-01, -8.8047e-01,\n",
       "         -9.1329e-01, -2.2054e-01,  9.9995e-01,  9.9985e-01, -9.9992e-01,\n",
       "         -9.9973e-01, -4.1531e-01,  5.9139e-02, -7.8567e-01,  9.9938e-01,\n",
       "          1.0516e-01, -9.9878e-01,  2.1823e-01,  1.8433e-01, -2.5135e-01,\n",
       "         -8.3777e-01,  1.0000e+00, -9.9484e-01,  1.0000e+00, -1.0000e+00,\n",
       "         -9.6393e-01,  1.9465e-01,  9.9998e-01, -9.9999e-01, -1.9518e-01,\n",
       "          9.9966e-01, -1.0000e+00, -2.9225e-02, -9.4787e-01,  8.8237e-01,\n",
       "         -3.2163e-02,  7.1631e-02,  7.8673e-01, -9.9974e-01,  1.6660e-01,\n",
       "         -9.9982e-01,  9.5086e-01,  9.9166e-01, -9.9993e-01, -6.1025e-01,\n",
       "         -9.9999e-01, -5.4383e-02,  6.0816e-02, -9.9975e-01,  9.9869e-01,\n",
       "          9.9999e-01, -4.4476e-02,  8.5795e-01, -9.9980e-01,  4.4504e-03,\n",
       "         -3.0101e-01, -9.8803e-01,  5.4812e-02, -9.9990e-01,  9.6314e-01,\n",
       "         -9.9127e-01,  9.9875e-01, -1.0000e+00,  9.8999e-01,  9.9710e-01,\n",
       "         -3.8264e-02, -6.5084e-01,  3.6452e-02,  4.2460e-01, -9.9999e-01,\n",
       "         -4.0223e-02, -9.9980e-01, -9.9983e-01,  2.8015e-01,  9.9988e-01,\n",
       "          9.9221e-01,  2.0411e-01,  9.9606e-01, -9.9796e-01, -2.8133e-02,\n",
       "          3.2979e-01,  6.6948e-01,  1.0000e+00, -9.9960e-01, -9.9993e-01,\n",
       "          9.9783e-01, -9.9996e-01, -9.9582e-01,  1.0000e+00,  1.0808e-01,\n",
       "          9.9989e-01, -2.8597e-02, -9.9971e-01,  1.2306e-01,  1.1798e-01,\n",
       "          9.9988e-01, -6.1640e-02, -1.1223e-01,  9.9997e-01, -1.1004e-01,\n",
       "          4.9045e-02, -6.0948e-01,  9.8479e-01, -2.3674e-01, -1.3137e-01,\n",
       "          9.9882e-01, -9.8893e-01, -9.9954e-01, -9.9989e-01,  1.8203e-01,\n",
       "          2.8674e-02,  4.0661e-02,  4.1384e-02,  8.4515e-01,  9.9998e-01,\n",
       "         -9.9956e-01,  9.9718e-01, -1.0000e+00, -9.9996e-01,  3.4787e-02,\n",
       "          1.6964e-01,  9.9935e-01, -9.4625e-02, -9.9383e-01,  1.3268e-01,\n",
       "         -9.8623e-01,  9.9770e-01, -9.9977e-01,  9.7668e-01, -1.0568e-01,\n",
       "          2.1731e-01,  9.9997e-01,  9.9913e-01, -3.6219e-02, -9.9880e-01,\n",
       "         -7.8271e-01, -2.8410e-02, -9.9888e-01,  9.9994e-01,  1.5550e-01,\n",
       "          1.9994e-01,  8.1395e-02,  1.4418e-03, -9.9998e-01, -9.9339e-01,\n",
       "          1.2288e-01,  9.9966e-01, -9.9993e-01,  9.9627e-01, -9.8925e-01,\n",
       "          9.9995e-01,  9.9989e-01,  1.0000e+00,  8.2978e-02,  9.9106e-01,\n",
       "         -9.9995e-01, -9.9636e-01,  9.7994e-01,  9.9933e-01,  1.0000e+00,\n",
       "          9.9820e-01,  9.7212e-01,  4.1774e-02, -9.9998e-01,  9.5319e-01,\n",
       "         -2.1234e-01, -2.5803e-01,  4.4171e-02, -9.7493e-01, -9.9998e-01,\n",
       "          9.9999e-01, -9.9999e-01, -9.9998e-01, -9.7688e-01, -9.9998e-01,\n",
       "          9.9955e-01,  6.3524e-01,  9.9913e-01,  8.2837e-01, -9.9992e-01,\n",
       "         -9.9911e-01, -7.4414e-02, -9.8721e-01, -9.9601e-01,  7.3158e-02,\n",
       "         -1.0000e+00,  4.0978e-02,  1.5363e-01, -9.5962e-01, -9.4314e-02,\n",
       "         -9.0655e-01, -1.2196e-01,  9.9709e-01,  6.6145e-01,  9.8879e-01,\n",
       "         -9.9556e-01, -9.9926e-01,  1.6966e-01, -1.0000e+00,  9.9555e-01,\n",
       "          9.9994e-01, -1.2530e-01,  9.5008e-01, -9.6306e-01,  1.6588e-01,\n",
       "         -1.0000e+00, -1.0000e+00,  9.7010e-01,  9.9986e-01,  3.6407e-03,\n",
       "         -9.9972e-01,  3.5594e-02, -9.9921e-01, -1.7513e-01,  9.5917e-01,\n",
       "          9.9811e-01, -9.9906e-01,  9.9963e-01, -9.9253e-01, -2.0149e-02,\n",
       "          9.9336e-01, -1.0000e+00,  8.5619e-01, -9.9406e-01,  1.0000e+00,\n",
       "         -1.0000e+00,  9.9858e-01, -4.8630e-01, -1.0990e-01, -1.3152e-02,\n",
       "          8.8953e-01, -9.9992e-01, -2.2119e-01,  9.9139e-01,  9.8939e-01,\n",
       "          6.7305e-03,  9.9942e-01, -1.8233e-01]], grad_fn=<TanhBackward0>), hidden_states=None, past_key_values=None, attentions=(tensor([[[[4.7840e-01, 3.7087e-04, 1.6194e-04,  ..., 1.4241e-04,\n",
       "           4.1823e-04, 5.1813e-01],\n",
       "          [4.8793e-03, 8.4205e-02, 1.0109e-01,  ..., 1.6317e-01,\n",
       "           1.3569e-01, 3.8025e-03],\n",
       "          [1.6934e-02, 9.4463e-02, 1.6830e-01,  ..., 1.0398e-01,\n",
       "           2.0319e-01, 3.0596e-03],\n",
       "          ...,\n",
       "          [2.6368e-02, 8.2262e-02, 4.2802e-02,  ..., 1.0264e-01,\n",
       "           2.0798e-01, 1.0292e-02],\n",
       "          [6.1473e-02, 5.6857e-02, 4.2318e-02,  ..., 7.1384e-02,\n",
       "           3.6086e-01, 2.4631e-02],\n",
       "          [4.6689e-01, 8.9996e-04, 4.2500e-04,  ..., 2.7257e-04,\n",
       "           1.1381e-03, 5.2718e-01]],\n",
       "\n",
       "         [[9.8990e-01, 2.5772e-05, 2.0234e-04,  ..., 5.2056e-05,\n",
       "           1.0738e-03, 3.7625e-03],\n",
       "          [1.7755e-02, 1.2910e-04, 9.8209e-01,  ..., 2.9791e-10,\n",
       "           2.1697e-06, 2.7124e-07],\n",
       "          [2.1831e-02, 9.7443e-01, 1.5326e-04,  ..., 1.2469e-08,\n",
       "           7.8246e-08, 1.0876e-04],\n",
       "          ...,\n",
       "          [1.1508e-02, 3.0013e-09, 4.1517e-08,  ..., 1.5005e-04,\n",
       "           3.2545e-03, 1.7668e-04],\n",
       "          [2.4576e-01, 2.9066e-06, 6.4727e-08,  ..., 3.6932e-03,\n",
       "           4.7574e-04, 7.4964e-01],\n",
       "          [5.9086e-01, 1.3709e-07, 7.4794e-05,  ..., 2.6531e-05,\n",
       "           4.0552e-01, 2.6671e-03]],\n",
       "\n",
       "         [[1.6501e-01, 9.1225e-02, 2.5748e-02,  ..., 5.6965e-02,\n",
       "           9.7241e-02, 2.3059e-01],\n",
       "          [4.7152e-01, 3.2132e-01, 3.1023e-02,  ..., 8.9475e-03,\n",
       "           1.6842e-02, 9.4982e-02],\n",
       "          [4.8925e-01, 1.7736e-01, 1.9220e-01,  ..., 4.7800e-03,\n",
       "           1.3740e-02, 6.2175e-02],\n",
       "          ...,\n",
       "          [1.5545e-01, 6.1267e-02, 5.8145e-02,  ..., 6.8622e-02,\n",
       "           2.9527e-02, 2.9667e-02],\n",
       "          [5.5324e-02, 4.2133e-02, 2.3946e-02,  ..., 4.4306e-02,\n",
       "           8.8572e-02, 8.7771e-02],\n",
       "          [3.6468e-02, 3.9798e-02, 4.4089e-02,  ..., 8.2621e-02,\n",
       "           1.4462e-01, 1.2662e-01]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[7.0319e-01, 2.1875e-02, 4.0400e-02,  ..., 2.3610e-02,\n",
       "           2.3715e-02, 7.4948e-02],\n",
       "          [4.0515e-01, 3.2106e-01, 1.1058e-02,  ..., 1.2410e-02,\n",
       "           1.7615e-02, 1.0445e-01],\n",
       "          [5.8258e-01, 5.5240e-03, 5.3974e-03,  ..., 1.7208e-02,\n",
       "           2.5468e-02, 2.7251e-01],\n",
       "          ...,\n",
       "          [3.7059e-01, 4.3855e-02, 4.8776e-02,  ..., 2.4265e-01,\n",
       "           2.1269e-02, 7.7568e-02],\n",
       "          [1.2533e-01, 1.6208e-02, 3.0249e-02,  ..., 2.5662e-02,\n",
       "           4.0757e-01, 2.9812e-01],\n",
       "          [4.2679e-01, 4.2644e-02, 8.9271e-02,  ..., 1.3507e-02,\n",
       "           2.0495e-02, 1.4134e-01]],\n",
       "\n",
       "         [[9.7357e-01, 5.4136e-03, 3.0843e-03,  ..., 1.6786e-03,\n",
       "           2.3268e-03, 5.4027e-03],\n",
       "          [7.9612e-03, 7.2660e-02, 4.8077e-01,  ..., 4.1847e-03,\n",
       "           1.9911e-03, 6.7627e-03],\n",
       "          [1.4957e-03, 7.2041e-03, 1.7663e-02,  ..., 6.2025e-04,\n",
       "           4.2359e-03, 2.2025e-03],\n",
       "          ...,\n",
       "          [1.7128e-03, 4.9460e-04, 1.8233e-04,  ..., 6.1111e-03,\n",
       "           8.0681e-01, 1.7855e-01],\n",
       "          [2.0596e-02, 1.4022e-03, 1.1201e-03,  ..., 7.0329e-03,\n",
       "           1.0126e-01, 8.6120e-01],\n",
       "          [9.9354e-01, 6.1979e-05, 1.0303e-04,  ..., 2.0507e-04,\n",
       "           6.5383e-04, 5.1244e-03]],\n",
       "\n",
       "         [[4.2150e-01, 2.4260e-02, 1.9585e-02,  ..., 1.5905e-02,\n",
       "           3.2343e-02, 3.2890e-01],\n",
       "          [7.5474e-01, 1.1008e-01, 2.2799e-03,  ..., 4.7717e-03,\n",
       "           1.6809e-02, 6.7962e-03],\n",
       "          [4.3970e-02, 9.3117e-01, 6.5203e-03,  ..., 1.7745e-04,\n",
       "           1.1036e-03, 1.4444e-03],\n",
       "          ...,\n",
       "          [4.3447e-02, 1.1219e-03, 7.6249e-05,  ..., 2.7540e-02,\n",
       "           1.1937e-02, 6.7831e-03],\n",
       "          [3.4735e-01, 1.1213e-02, 1.3291e-02,  ..., 2.7917e-01,\n",
       "           1.3353e-01, 5.0572e-02],\n",
       "          [7.1003e-02, 1.5132e-03, 7.3035e-04,  ..., 2.2069e-02,\n",
       "           3.9020e-01, 5.0058e-01]]]], grad_fn=<SoftmaxBackward0>), tensor([[[[4.3653e-01, 1.2017e-02, 5.9486e-03,  ..., 6.0889e-03,\n",
       "           6.2510e-02, 4.1911e-01],\n",
       "          [4.9485e-01, 1.4810e-02, 3.5573e-03,  ..., 2.6525e-06,\n",
       "           7.7276e-03, 4.7143e-01],\n",
       "          [1.3883e-02, 9.7162e-01, 1.4196e-05,  ..., 1.7454e-07,\n",
       "           8.6182e-06, 1.3725e-02],\n",
       "          ...,\n",
       "          [3.9525e-02, 7.2212e-06, 4.0403e-06,  ..., 6.8149e-06,\n",
       "           2.1746e-03, 3.7589e-02],\n",
       "          [3.7276e-01, 8.1322e-04, 4.1047e-04,  ..., 1.7777e-01,\n",
       "           4.8129e-02, 3.6578e-01],\n",
       "          [4.3777e-01, 1.1416e-02, 5.8509e-03,  ..., 5.9089e-03,\n",
       "           6.3776e-02, 4.2000e-01]],\n",
       "\n",
       "         [[4.6678e-01, 1.0235e-02, 6.1202e-03,  ..., 6.2378e-03,\n",
       "           1.0923e-02, 4.4965e-01],\n",
       "          [2.9057e-01, 1.4343e-02, 1.1761e-02,  ..., 3.3617e-02,\n",
       "           1.1928e-01, 2.9151e-01],\n",
       "          [2.3338e-01, 2.3156e-02, 8.7420e-03,  ..., 6.2520e-02,\n",
       "           1.5599e-01, 2.3423e-01],\n",
       "          ...,\n",
       "          [3.0643e-01, 2.5059e-02, 1.2080e-02,  ..., 3.9024e-02,\n",
       "           1.5327e-01, 3.0960e-01],\n",
       "          [1.2985e-01, 2.9326e-02, 1.4902e-02,  ..., 9.5539e-02,\n",
       "           2.5447e-01, 1.3206e-01],\n",
       "          [4.6893e-01, 9.5607e-03, 5.7683e-03,  ..., 5.9750e-03,\n",
       "           1.0103e-02, 4.5207e-01]],\n",
       "\n",
       "         [[4.9365e-01, 4.6236e-03, 4.0628e-03,  ..., 1.9692e-03,\n",
       "           4.7649e-03, 4.5183e-01],\n",
       "          [4.0014e-01, 2.0960e-01, 9.1870e-03,  ..., 1.3033e-03,\n",
       "           6.6394e-04, 3.6989e-01],\n",
       "          [3.9738e-01, 1.1984e-01, 8.2242e-02,  ..., 1.2187e-03,\n",
       "           2.4011e-03, 3.7076e-01],\n",
       "          ...,\n",
       "          [2.4969e-01, 4.7297e-03, 2.8019e-04,  ..., 7.1960e-02,\n",
       "           4.7627e-03, 2.3028e-01],\n",
       "          [2.5165e-01, 1.1145e-03, 1.3995e-03,  ..., 1.8613e-03,\n",
       "           4.7934e-01, 2.3995e-01],\n",
       "          [4.9327e-01, 4.8001e-03, 4.1925e-03,  ..., 2.0644e-03,\n",
       "           4.7635e-03, 4.5095e-01]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[4.5285e-01, 9.8046e-03, 1.9008e-02,  ..., 1.4504e-02,\n",
       "           2.5172e-02, 4.2710e-01],\n",
       "          [2.7714e-01, 1.5313e-02, 2.6586e-02,  ..., 1.8827e-02,\n",
       "           1.9736e-02, 2.7695e-01],\n",
       "          [2.5708e-01, 7.3269e-03, 1.1370e-02,  ..., 1.5365e-02,\n",
       "           2.2513e-02, 2.6464e-01],\n",
       "          ...,\n",
       "          [4.4378e-01, 2.6481e-03, 2.3780e-03,  ..., 2.4616e-02,\n",
       "           4.9823e-02, 4.5435e-01],\n",
       "          [4.2933e-01, 1.6639e-02, 7.7778e-03,  ..., 1.6536e-02,\n",
       "           4.9764e-02, 4.2995e-01],\n",
       "          [4.5464e-01, 9.4518e-03, 1.8229e-02,  ..., 1.3868e-02,\n",
       "           2.4021e-02, 4.2919e-01]],\n",
       "\n",
       "         [[4.9040e-01, 4.7597e-03, 2.2697e-03,  ..., 1.4708e-03,\n",
       "           5.1065e-03, 4.6151e-01],\n",
       "          [4.4746e-01, 1.8176e-02, 1.2351e-02,  ..., 4.3363e-03,\n",
       "           9.6561e-03, 4.4103e-01],\n",
       "          [3.5675e-01, 4.1090e-02, 2.7275e-02,  ..., 1.2361e-02,\n",
       "           3.0071e-02, 3.4211e-01],\n",
       "          ...,\n",
       "          [1.8479e-01, 5.4278e-02, 3.6304e-02,  ..., 1.0251e-02,\n",
       "           8.2401e-02, 1.8141e-01],\n",
       "          [1.3622e-01, 5.0552e-02, 4.2730e-02,  ..., 1.3541e-02,\n",
       "           9.1346e-02, 1.3696e-01],\n",
       "          [4.8996e-01, 4.8203e-03, 2.2924e-03,  ..., 1.5024e-03,\n",
       "           5.0243e-03, 4.6148e-01]],\n",
       "\n",
       "         [[4.7184e-01, 6.5952e-03, 1.4942e-02,  ..., 2.0032e-03,\n",
       "           8.9569e-03, 4.5821e-01],\n",
       "          [2.8745e-01, 6.4402e-02, 5.5145e-02,  ..., 5.8005e-02,\n",
       "           3.4206e-02, 2.8082e-01],\n",
       "          [3.6048e-01, 8.8081e-02, 3.8540e-02,  ..., 1.2227e-02,\n",
       "           1.3103e-02, 3.4988e-01],\n",
       "          ...,\n",
       "          [2.8444e-01, 2.5244e-01, 1.4736e-02,  ..., 8.4384e-03,\n",
       "           1.0795e-02, 2.8038e-01],\n",
       "          [2.2920e-01, 8.9625e-02, 2.8974e-02,  ..., 1.2092e-02,\n",
       "           3.2623e-02, 2.2609e-01],\n",
       "          [4.7172e-01, 6.4383e-03, 1.5023e-02,  ..., 1.9758e-03,\n",
       "           8.8361e-03, 4.5832e-01]]]], grad_fn=<SoftmaxBackward0>), tensor([[[[3.2780e-01, 2.6365e-02, 3.2871e-02,  ..., 9.6133e-03,\n",
       "           1.0898e-01, 3.2316e-01],\n",
       "          [4.5742e-01, 3.5140e-02, 2.2830e-02,  ..., 2.0939e-03,\n",
       "           4.4799e-03, 4.5110e-01],\n",
       "          [4.8155e-01, 1.6324e-02, 9.9995e-03,  ..., 1.1973e-03,\n",
       "           3.4324e-03, 4.7338e-01],\n",
       "          ...,\n",
       "          [2.7153e-01, 6.4185e-02, 2.8570e-02,  ..., 4.9459e-03,\n",
       "           6.7410e-03, 2.6692e-01],\n",
       "          [1.8439e-01, 3.0287e-02, 2.7450e-02,  ..., 1.5050e-02,\n",
       "           7.3344e-02, 1.8221e-01],\n",
       "          [3.2881e-01, 2.6313e-02, 3.2750e-02,  ..., 9.6148e-03,\n",
       "           1.0816e-01, 3.2417e-01]],\n",
       "\n",
       "         [[1.8750e-02, 6.2589e-02, 2.2339e-02,  ..., 5.5634e-02,\n",
       "           3.3249e-01, 1.8649e-02],\n",
       "          [3.1858e-01, 8.1700e-02, 2.1848e-01,  ..., 6.8917e-03,\n",
       "           1.1892e-03, 3.1426e-01],\n",
       "          [8.9934e-02, 8.1036e-01, 3.1088e-03,  ..., 8.8777e-05,\n",
       "           1.2451e-04, 8.9184e-02],\n",
       "          ...,\n",
       "          [4.2126e-01, 9.0581e-03, 1.5826e-03,  ..., 6.9298e-03,\n",
       "           5.4295e-03, 4.1756e-01],\n",
       "          [8.6953e-02, 1.5984e-01, 1.7643e-02,  ..., 3.6727e-01,\n",
       "           2.2709e-02, 8.6116e-02],\n",
       "          [1.8879e-02, 6.2910e-02, 2.2460e-02,  ..., 5.5693e-02,\n",
       "           3.3281e-01, 1.8778e-02]],\n",
       "\n",
       "         [[4.4325e-01, 1.9266e-03, 7.8245e-04,  ..., 1.6809e-03,\n",
       "           8.7054e-02, 4.4051e-01],\n",
       "          [4.3983e-01, 3.2083e-02, 3.4782e-02,  ..., 4.1581e-03,\n",
       "           7.0871e-04, 4.3195e-01],\n",
       "          [4.4349e-01, 1.5942e-02, 5.0625e-02,  ..., 1.1889e-02,\n",
       "           1.2361e-03, 4.3593e-01],\n",
       "          ...,\n",
       "          [4.4214e-01, 3.2768e-03, 8.1812e-03,  ..., 5.7090e-02,\n",
       "           2.4541e-03, 4.3559e-01],\n",
       "          [3.9564e-01, 2.1476e-02, 2.0576e-02,  ..., 3.3113e-02,\n",
       "           2.6663e-02, 3.9208e-01],\n",
       "          [4.4326e-01, 1.9491e-03, 7.8429e-04,  ..., 1.6953e-03,\n",
       "           8.6944e-02, 4.4053e-01]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[4.1120e-02, 1.0956e-01, 6.5788e-02,  ..., 1.1601e-01,\n",
       "           1.0126e-01, 4.0739e-02],\n",
       "          [3.2548e-01, 3.4923e-02, 4.0616e-02,  ..., 2.6153e-02,\n",
       "           3.6910e-03, 3.2213e-01],\n",
       "          [3.1822e-01, 3.9943e-02, 1.7110e-02,  ..., 1.6980e-02,\n",
       "           5.5877e-03, 3.1424e-01],\n",
       "          ...,\n",
       "          [4.9317e-01, 5.5938e-04, 3.0690e-04,  ..., 2.8318e-03,\n",
       "           4.3566e-03, 4.8936e-01],\n",
       "          [4.5793e-01, 7.9626e-03, 7.1378e-03,  ..., 1.5483e-02,\n",
       "           2.1822e-02, 4.5540e-01],\n",
       "          [4.1555e-02, 1.0989e-01, 6.5843e-02,  ..., 1.1542e-01,\n",
       "           1.0105e-01, 4.1171e-02]],\n",
       "\n",
       "         [[4.4587e-02, 3.3572e-02, 6.0334e-02,  ..., 1.3603e-01,\n",
       "           2.4219e-01, 4.4369e-02],\n",
       "          [4.8535e-01, 2.0157e-02, 6.9701e-03,  ..., 4.0497e-04,\n",
       "           6.0358e-04, 4.8044e-01],\n",
       "          [4.4549e-01, 6.5994e-02, 3.4386e-02,  ..., 6.7095e-04,\n",
       "           1.3239e-03, 4.3875e-01],\n",
       "          ...,\n",
       "          [1.5787e-01, 1.8229e-02, 7.8358e-03,  ..., 9.8154e-03,\n",
       "           8.2203e-03, 1.5449e-01],\n",
       "          [1.4511e-01, 3.5189e-02, 2.2744e-02,  ..., 2.9217e-02,\n",
       "           4.7360e-02, 1.4272e-01],\n",
       "          [4.4703e-02, 3.3379e-02, 6.0042e-02,  ..., 1.3702e-01,\n",
       "           2.4263e-01, 4.4484e-02]],\n",
       "\n",
       "         [[1.6944e-01, 3.7057e-02, 2.3734e-02,  ..., 4.7385e-02,\n",
       "           2.5237e-01, 1.6619e-01],\n",
       "          [3.4206e-01, 9.6887e-03, 1.2336e-01,  ..., 6.0542e-03,\n",
       "           8.7858e-03, 3.3832e-01],\n",
       "          [3.6341e-01, 5.1030e-03, 7.5189e-03,  ..., 7.5258e-04,\n",
       "           3.7137e-03, 3.5974e-01],\n",
       "          ...,\n",
       "          [4.8866e-01, 2.6738e-04, 3.1088e-04,  ..., 9.8990e-04,\n",
       "           1.7911e-02, 4.8671e-01],\n",
       "          [4.0732e-01, 3.8137e-02, 9.6832e-03,  ..., 4.4490e-02,\n",
       "           2.2997e-02, 4.0793e-01],\n",
       "          [1.7047e-01, 3.6989e-02, 2.3646e-02,  ..., 4.6833e-02,\n",
       "           2.5233e-01, 1.6721e-01]]]], grad_fn=<SoftmaxBackward0>)), cross_attentions=None)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 62
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:29.213569Z",
     "start_time": "2025-09-05T14:11:29.209321Z"
    }
   },
   "source": [
    "output.last_hidden_state.size()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 12, 768])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 63
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:30.631902Z",
     "start_time": "2025-09-05T14:11:30.626962Z"
    }
   },
   "source": [
    "len(inputs[\"input_ids\"][0])"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 64
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 带Model Head的模型调用"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:31.990694Z",
     "start_time": "2025-09-05T14:11:31.986189Z"
    }
   },
   "source": "from transformers import AutoModelForSequenceClassification",
   "outputs": [],
   "execution_count": 65
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:11:33.368213Z",
     "start_time": "2025-09-05T14:11:33.295825Z"
    }
   },
   "source": "clz_model = AutoModelForSequenceClassification.from_pretrained(\"rbt3\", num_labels = 10)",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "execution_count": 66
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:13:23.581354Z",
     "start_time": "2025-09-05T14:13:23.565861Z"
    }
   },
   "source": [
    "clz_model(**inputs)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequenceClassifierOutput(loss=None, logits=tensor([[ 0.5153, -0.0834,  0.3421,  0.1638,  0.2085,  0.4270, -0.2053,  0.1997,\n",
       "         -0.4440,  1.4064]], grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 71
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T14:13:22.299832Z",
     "start_time": "2025-09-05T14:13:22.295657Z"
    }
   },
   "source": [
    "clz_model.config.num_labels"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 70
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
